import requests
from lxml import etree

from multiprocessing.dummy import Pool
headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
}
urls = [] # 存储所有视频链接和名字

def get_video_data(dic):
    url = dic['url']# 视频链接
    video_data = requests.get(url=url, headers=headers).content
    print(dic['name'], '正在下载')
    with open(dic['name'], 'wb') as fp:
        fp.write(video_data)
        print(dic['name'], '下载完成')

if __name__ == '__main__':
    url = 'https://www.pearvideo.com/category_5'
    page_text = requests.get(url=url, headers=headers).text

    tree = etree.HTML(page_text)
    li_list = tree.xpath('//ul[@class="listvideo-list clearfix"]/li')
    for li in li_list:
        detail_url = 'https://www.pearvideo.com/' + li.xpath('./div/a/@href')[0]
        name = li.xpath('./div/a/div[2]/text()')[0] + '.mp4'
        a_href_id = str(li.xpath('./div/a/@href')[0]).split('_')[1]

        mp4_headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
            'Referer': detail_url
        }
        mp4_ajax_url = 'https://www.pearvideo.com/videoStatus.jsp'
        param = {
            'contId': a_href_id,
            'mrd': '0'
        }

        response = requests.get(url=mp4_ajax_url, params=param, headers=mp4_headers).json()
        video_url = response['videoInfo']['videos']['srcUrl']

        dic = {
            'name':name,
            'url':video_url
        }
        urls.append(dic)
# 线程池原则：线程池处理的是阻塞且耗时的操作
# 使用线程池对视频数据进行请求
pool = Pool(4)
pool.map(get_video_data, urls)

# 线程池使用结束后，关闭线程池
pool.close()
pool.join()